from typing import List
from utils.database import Kafka
from items import RequestItem, ResponseItem, DataBaseItem
from config import settings
from spider_result_processor import spider_result_processor


class HTTPBinGetItem(DataBaseItem):
    origin: str


def start():
    for i in range(16):
        Kafka.send_req_item(
            RequestItem(
                url="https://httpbin.org/get",
                proxies=True,
                parse_callback="workers.httpbin.parse"
                               "",
                data_callback="workers.httpbin.save"
            )
        )


def parse(resp_item: ResponseItem) -> DataBaseItem:
    data = resp_item.data
    yield HTTPBinGetItem(origin=data['origin'])


def save(items: List[HTTPBinGetItem]):
    for item in items:
        print(f"我的IP是{item.origin}")


if __name__ == '__main__':
    start()
    spider_result_processor(
        topic=f"{settings.kafka.consumer.topic}_result",
        group_id="123",
        timeout=10000,
        max_count=100
    )
